import pandas as pd
from sklearn.model_selection import train_test_split


data = pd.read_csv('cs-training.csv')

#去重
data = data.drop_duplicates()
data.reset_index(inplace=True, drop=True)

#对于家庭成员来说，只有2%缺失，可以用均值代替
data['NumberOfDependents'] = data['NumberOfDependents'].fillna(int(data['NumberOfDependents'].mean()))

#对于收入，用随机森林进行填补
def fill_missing_value(col):
    from sklearn.ensemble import RandomForestRegressor
    #有数据与无数据
    with_data = data[data[col].notnull()]
    na_data = data[data[col].isnull()]
    #除去指定列外的其他列为X
    col_x = [i for i in with_data.columns if i != col]
    X = with_data[col_x]
    y = with_data[col]
    
    rfr = RandomForestRegressor(n_estimators=100, max_depth=3)
    rfr.fit(X, y)
    y_pre = rfr.predict(na_data[col_x])
    data.loc[data[col].isnull(), col] = y_pre
    
fill_missing_value('MonthlyIncome')


##处理异常值, 画箱线图判定
#statstic = data.describe([.01, .1, .25, .5, .75, .95]).T
#去掉年龄为0的
data = data[data.age > 0]
#又225个违约次数超过95次，其他140,000+都低于15次，应该是错误数据，去掉。
data = data[data['NumberOfTime30-59DaysPastDueNotWorse'] < 90]

# #仅有554个RevolvingUtilizationOfUnsecuredLines 大于2， 判定为异常值。去除
# data = data[data['RevolvingUtilizationOfUnsecuredLines'] <= 2]
# #仅有5000条数据debratio大于5000，判定为异常值，去掉
# data = data[data['DebtRatio'] <= 5000]
# #仅有9条数据monthlyincom大于500,000
# data = data[data['MonthlyIncome'] <= 500000]
# #仅有661条NumberRealEstateLoansOrLines大于6
# data = data[data['NumberRealEstateLoansOrLines'] <= 6]
# #家庭成员大于8的仅有6家
# data = data[data['NumberOfDependents'] <= 8]

#reset_index
data.reset_index(inplace=True, drop=True)


#解决样本不均衡不问题
from imblearn.over_sampling import SMOTE

X, y = data.iloc[:, 1:], data.iloc[:, 0]
sm = SMOTE(random_state=42)
X, y = sm.fit_sample(X, y)

###拆分训练集
X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=.3, random_state=420)

###保存处理好的数据
model_set = pd.concat([y_train, X_train], axis=1)
test_set = pd.concat([y_test, X_test], axis=1)

model_set.to_csv('model_set.csv', encoding='utf-8', index=False)
test_set.to_csv('test_set.csv', encoding='utf-8', index=False)